{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tsfresh提取特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tsfresh import select_features\n",
    "from tsfresh import extract_features\n",
    "from tsfresh.feature_extraction import EfficientFCParameters\n",
    "from tsfresh.feature_extraction import ComprehensiveFCParameters\n",
    "from joblib import Parallel, delayed\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 通过运行ComprehensiveFCParameters()可以得到完整的特征设置的字典\n",
    "minimal_parameters = {\n",
    "     'length': None,\n",
    "     'abs_energy': None,\n",
    "    \n",
    "    'mean':None,\n",
    "    'mean_abs_change': None,\n",
    "    'mean_change': None,\n",
    "     'variance': None,\n",
    "     'skewness': None,\n",
    "     'kurtosis': None,\n",
    "\n",
    "}\n",
    "settings = minimal_parameters\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_tsfresh_feature_basis(sensor,settings):\n",
    "    '''\n",
    "    描述：\n",
    "        settings，使用tsfresh提取指定特征\n",
    "    参数：\n",
    "        sensor：\n",
    "        settings(dict): 想要提取的特征\n",
    "    '''\n",
    "    if 'vibration_1' in settings.keys():\n",
    "        extracted_features = extract_features(sensor, column_id='id', column_sort='sort_col', n_jobs=48, \n",
    "                                          kind_to_fc_parameters=settings, disable_progressbar = True)\n",
    "    else:\n",
    "        extracted_features = extract_features(sensor, column_id='id', column_sort='sort_col', n_jobs=48, \n",
    "                                          default_fc_parameters=settings, disable_progressbar = False)\n",
    "    return extracted_features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_tsfresh_feature_parallel(data_no, csv_nos, settings):\n",
    "    '''\n",
    "    描述：\n",
    "        并行提取多个senosr文件的tsfresh特征\n",
    "    参数：\n",
    "        data_no：第几个plc\n",
    "        csv_nos：plc对应的sensor文件个数\n",
    "    '''\n",
    "    \n",
    "    input_dir = './sensors_ad/%s/'%data_no\n",
    "    output_dir = './sensors_tsfresh_comprehensive/%s/'%data_no\n",
    "    \n",
    "    if not os.path.exists('./sensors_tsfresh_comprehensive/'):\n",
    "        os.mkdir('./sensors_tsfresh_comprehensive')\n",
    "    if not os.path.exists('./sensors_tsfresh_comprehensive/%s'%data_no):\n",
    "        os.mkdir('./sensors_tsfresh_comprehensive/%s'%data_no)\n",
    "    \n",
    "    def basis_func(idx):\n",
    "        sensor = joblib.load(input_dir + '%d.lz4'%idx)\n",
    "        tmp = gen_tsfresh_feature_basis(sensor, settings)\n",
    "        joblib.dump(tmp, output_dir+'%d.lz4'%idx, compress='lz4')\n",
    "        \n",
    "    Parallel(n_jobs=1,verbose=10)(delayed(basis_func)(i) for i in range(1,csv_nos+1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "Feature Extraction: 100%|██████████| 237/237 [00:01<00:00, 187.26it/s]\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    6.5s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 236/236 [00:00<00:00, 250.05it/s]\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:   13.3s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 237/237 [00:00<00:00, 247.92it/s]\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   19.2s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 232/232 [00:00<00:00, 241.79it/s]\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:   24.9s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 238/238 [00:00<00:00, 262.97it/s]\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   30.5s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 235/235 [00:00<00:00, 236.38it/s]\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:   36.3s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 237/237 [00:01<00:00, 228.53it/s]\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:   42.5s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 236/236 [00:00<00:00, 259.40it/s]\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:   48.1s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 233/233 [00:00<00:00, 243.71it/s]\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:   53.4s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 233/233 [00:00<00:00, 242.89it/s]\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   59.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "Feature Extraction: 100%|██████████| 234/234 [00:00<00:00, 264.70it/s]\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    4.9s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 233/233 [00:00<00:00, 246.83it/s]\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    9.7s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 233/233 [00:00<00:00, 234.96it/s]\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   14.6s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 236/236 [00:00<00:00, 259.36it/s]\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:   19.0s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 231/231 [00:00<00:00, 290.70it/s]\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   23.6s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 239/239 [00:01<00:00, 238.08it/s]\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:   28.4s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 234/234 [00:00<00:00, 272.07it/s]\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:   33.2s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 234/234 [00:00<00:00, 283.00it/s]\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:   37.9s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 233/233 [00:00<00:00, 317.92it/s]\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:   42.3s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 240/240 [00:00<00:00, 279.08it/s]\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   47.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "Feature Extraction: 100%|██████████| 232/232 [00:00<00:00, 251.19it/s]\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    4.9s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 236/236 [00:00<00:00, 236.81it/s]\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    9.9s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 235/235 [00:01<00:00, 230.74it/s]\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   14.9s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 235/235 [00:00<00:00, 242.76it/s]\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:   19.8s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 237/237 [00:01<00:00, 220.09it/s]\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   24.8s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 235/235 [00:00<00:00, 235.21it/s]\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:   29.8s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 238/238 [00:01<00:00, 233.21it/s]\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:   36.6s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 238/238 [00:01<00:00, 236.96it/s]\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:   41.6s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 234/234 [00:00<00:00, 247.67it/s]\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:   46.3s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 239/239 [00:01<00:00, 231.30it/s]\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   51.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "Feature Extraction: 100%|██████████| 239/239 [00:01<00:00, 194.16it/s]\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    5.3s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 235/235 [00:01<00:00, 195.86it/s]\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:   10.7s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 239/239 [00:01<00:00, 200.25it/s]\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   16.0s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 240/240 [00:01<00:00, 179.08it/s]\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:   21.6s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 235/235 [00:01<00:00, 203.49it/s]\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   26.9s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 239/239 [00:01<00:00, 171.80it/s]\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:   32.4s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 237/237 [00:01<00:00, 181.03it/s]\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:   37.8s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 238/238 [00:01<00:00, 205.15it/s]\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:   43.1s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 237/237 [00:01<00:00, 211.31it/s]\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:   48.3s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 240/240 [00:01<00:00, 182.36it/s]\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   53.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "Feature Extraction: 100%|██████████| 235/235 [00:01<00:00, 183.18it/s]\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    4.8s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 235/235 [00:01<00:00, 187.68it/s]\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    9.2s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 237/237 [00:01<00:00, 186.44it/s]\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   13.9s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 240/240 [00:01<00:00, 192.90it/s]\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:   18.5s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 240/240 [00:01<00:00, 192.03it/s]\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   23.0s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 238/238 [00:01<00:00, 208.49it/s]\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:   27.5s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 238/238 [00:01<00:00, 193.90it/s]\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:   31.9s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 238/238 [00:01<00:00, 209.04it/s]\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:   36.4s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 237/237 [00:01<00:00, 192.80it/s]\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:   40.9s remaining:    0.0s\n",
      "Feature Extraction: 100%|██████████| 240/240 [00:01<00:00, 175.01it/s]\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   45.5s finished\n"
     ]
    }
   ],
   "source": [
    "gen_tsfresh_feature_parallel('01', 10, settings)\n",
    "gen_tsfresh_feature_parallel('02', 10 , settings)\n",
    "gen_tsfresh_feature_parallel('03', 10, settings)\n",
    "gen_tsfresh_feature_parallel('04', 10 , settings)\n",
    "gen_tsfresh_feature_parallel('05', 10, settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import joblib\n",
    "# test = joblib.load('./sensors_tsfresh_comprehensive/01/1.lz4')\n",
    "# test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
